{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 生成小样本数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "546394"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"train.txt\") as f1:\n",
    "    train = f1.readlines()\n",
    "train = [i for i in train if i.strip()!=\"\" and i[0]!=\"0\"]\n",
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "240671"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"dev.txt\") as f1:\n",
    "    dev = f1.readlines()\n",
    "dev = [i for i in dev if i.strip()!=\"\"]\n",
    "len(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "358435"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"test.txt\") as f1:\n",
    "    test = f1.readlines()\n",
    "test = [i for i in test if i.strip()!=\"\"]\n",
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71652"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([i for i in test if i[0]==\"0\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "def get_lines(len_line=20000, only_train=True, seed=123):\n",
    "    random.seed(seed)\n",
    "    random.shuffle(train)\n",
    "    random.shuffle(dev)\n",
    "    random.shuffle(test)\n",
    "    len_line_str = f\"_{len_line}\" if len_line!=None else \"\"\n",
    "    with open(f\"train{len_line_str}.txt\", \"w\", encoding=\"utf-8\") as f1:\n",
    "        for index, line in enumerate(train[:len_line]):\n",
    "            if index == 0 and \"1097:\" not in line:\n",
    "                line = line.strip() + \" 1097:0\\n\"\n",
    "            f1.write(line)\n",
    "    if only_train == False:\n",
    "        with open(f\"dev{len_line_str}.txt\", \"w\", encoding=\"utf-8\") as f1:\n",
    "            for index, line in enumerate(dev[:len_line]):\n",
    "                if index == 0 and \"1097:\" not in line:\n",
    "                    line = line.strip() + \" 1097:0\\n\"\n",
    "                f1.write(line)\n",
    "\n",
    "        if len_line!=None:\n",
    "            with open(f\"test{len_line_str}.txt\", \"w\", encoding=\"utf-8\") as f1:\n",
    "                for index, line in enumerate(test[:len_line]):\n",
    "                    if index == 0 and \"1097:\" not in line:\n",
    "                        line = line.strip() + \" 1097:0\\n\"\n",
    "                    f1.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 1.1G\n",
      "-rw-r--r-- 1 user63 users 7.8M Sep 11 12:59 dev_10000.txt\n",
      "-rw-r--r-- 1 user63 users  16M Sep 11 12:59 dev_20000.txt\n",
      "-rw-r--r-- 1 user63 users 3.9M Sep 11 12:59 dev_5000.txt\n",
      "-rw-r--r-- 1 user63 users  21K Sep  9 18:29 dev_demo.txt\n",
      "-rwxrwxr-x 1 user63 users 188M Sep 11 20:22 dev.txt\n",
      "-rwxrwxr-x 1 user63 users 110K Sep  2 17:57 keys.json\n",
      "-rw-r--r-- 1 user63 users 4.0K Sep 11 13:30 label_freq.json\n",
      "-rw-r--r-- 1 user63 users  207 Sep  9 11:12 Readme.md\n",
      "-rw-r--r-- 1 user63 users 7.8M Sep 11 12:59 test_10000.txt\n",
      "-rw-r--r-- 1 user63 users  16M Sep 11 12:59 test_20000.txt\n",
      "-rw-r--r-- 1 user63 users 3.9M Sep 11 10:36 test_5000_in_dev.txt\n",
      "-rw-r--r-- 1 user63 users 3.9M Sep 11 12:59 test_5000.txt\n",
      "-rw-r--r-- 1 user63 users  40K Sep  9 18:35 test_demo.txt\n",
      "-rwxrwxr-x 1 user63 users 281M Sep 11 13:07 test.txt\n",
      "-rw-r--r-- 1 user63 users  78M Sep 11 20:23 train_100000.txt\n",
      "-rw-r--r-- 1 user63 users 7.8M Sep 11 20:23 train_10000.txt\n",
      "-rw-r--r-- 1 user63 users  16M Sep 11 20:23 train_20000.txt\n",
      "-rw-r--r-- 1 user63 users 3.9M Sep 11 12:59 train_5000.txt\n",
      "-rwxrwxr-x 1 user63 users  21K Sep  9 18:26 train_demo.txt\n",
      "-rwxrwxr-x 1 user63 users 426M Sep 11 20:23 train.txt\n",
      "-rw-r--r-- 1 user63 users  24K Sep 11 19:12 确认首行有最大下标.ipynb\n"
     ]
    }
   ],
   "source": [
    "get_lines(100000)\n",
    "get_lines(10000)\n",
    "get_lines(20000)\n",
    "get_lines(None)\n",
    "!ls -hl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_dev = [item.split(\" \")[0] for item in dev]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = Counter(labels_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('5', 24013),\n",
       " ('4', 21188),\n",
       " ('1', 18074),\n",
       " ('8', 17576),\n",
       " ('22', 13787),\n",
       " ('20', 11043),\n",
       " ('29', 8224),\n",
       " ('33', 6761),\n",
       " ('0', 6645),\n",
       " ('15', 5040)]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 前3占比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "240671"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_freq = [(int(item), freq, freq/len(labels_dev)) for item,freq in c.most_common()]\n",
    "with open(\"label_freq.json\",\"w\") as f1:\n",
    "    json.dump(label_freq, f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 (5, 24013, 0.09977521180366558)\n",
      "2 (4, 21188, 0.0880371960061661)\n",
      "3 (1, 18074, 0.0750983708049578)\n",
      "4 (8, 17576, 0.07302915598472604)\n",
      "5 (22, 13787, 0.05728567214163734)\n",
      "6 (20, 11043, 0.045884215381163496)\n",
      "7 (29, 8224, 0.03417112988270294)\n",
      "8 (33, 6761, 0.028092291967042146)\n",
      "9 (0, 6645, 0.027610306185622695)\n",
      "10 (15, 5040, 0.020941451192707055)\n",
      "11 (25, 4355, 0.01809524205242842)\n",
      "12 (34, 3921, 0.01629195042194531)\n",
      "13 (35, 3543, 0.014721341582492282)\n",
      "14 (36, 3238, 0.013454051381346319)\n",
      "15 (23, 3099, 0.012876499453610945)\n",
      "16 (32, 2758, 0.01145962745823136)\n",
      "17 (6, 2637, 0.010956866427612798)\n",
      "18 (43, 2603, 0.010815594733058823)\n",
      "19 (3, 2564, 0.010653547789305733)\n",
      "20 (47, 2447, 0.010167406958046462)\n",
      "21 (12, 2297, 0.009544149482073038)\n",
      "22 (63, 2234, 0.009282381342164199)\n",
      "23 (7, 2226, 0.009249140943445616)\n",
      "24 (51, 2139, 0.00888765160738103)\n",
      "25 (39, 2096, 0.008708984464268649)\n",
      "26 (11, 2095, 0.008704829414428825)\n",
      "27 (30, 1877, 0.007799028549347449)\n",
      "28 (46, 1672, 0.00694724333218377)\n",
      "29 (86, 1652, 0.006864142335387313)\n",
      "30 (13, 1641, 0.006818436787149262)\n",
      "31 (57, 1611, 0.006693785291954577)\n",
      "32 (75, 1589, 0.006602374195478475)\n",
      "33 (42, 1497, 0.006220109610214774)\n",
      "34 (2, 1483, 0.006161938912457255)\n",
      "35 (49, 1454, 0.0060414424671023925)\n",
      "36 (19, 1435, 0.005962496520145759)\n",
      "37 (73, 1378, 0.005725658679275858)\n",
      "38 (69, 1378, 0.005725658679275858)\n",
      "39 (52, 1325, 0.005505441037765248)\n",
      "40 (27, 1320, 0.005484665788566134)\n",
      "41 (71, 1297, 0.005389099642250208)\n",
      "42 (65, 1294, 0.005376634492730741)\n",
      "43 (28, 1191, 0.004948664359228989)\n",
      "44 (81, 1138, 0.004728446717718379)\n",
      "45 (95, 1124, 0.0046702760199608595)\n",
      "46 (64, 1000, 0.0041550498398228285)\n",
      "47 (21, 950, 0.003947297347831687)\n",
      "48 (91, 882, 0.0036647539587237347)\n",
      "49 (18, 874, 0.0036315135600051523)\n",
      "50 (48, 874, 0.0036315135600051523)\n",
      "51 (76, 788, 0.003274179273780389)\n",
      "52 (103, 776, 0.003224318675702515)\n",
      "53 (80, 768, 0.0031910782769839323)\n",
      "54 (98, 765, 0.0031786131274644637)\n",
      "55 (78, 755, 0.0031370626290662357)\n",
      "56 (26, 719, 0.002987480834832614)\n",
      "57 (58, 718, 0.002983325784992791)\n",
      "58 (55, 668, 0.0027755732930016493)\n",
      "59 (66, 658, 0.0027340227946034213)\n",
      "60 (24, 652, 0.002709092495564484)\n",
      "61 (74, 651, 0.0027049374457246613)\n",
      "62 (84, 647, 0.0026883172463653703)\n",
      "63 (77, 646, 0.0026841621965255475)\n",
      "64 (45, 611, 0.002538735452131748)\n",
      "65 (97, 585, 0.002430704156296355)\n",
      "66 (17, 574, 0.0023849986080583035)\n",
      "67 (10, 571, 0.002372533458538835)\n",
      "68 (102, 555, 0.0023060526611016698)\n",
      "69 (9, 552, 0.0022935875115822012)\n",
      "70 (70, 543, 0.002256192063023796)\n",
      "71 (107, 540, 0.0022437269135043274)\n",
      "72 (93, 533, 0.0022146415646255675)\n",
      "73 (116, 521, 0.0021647809665476937)\n",
      "74 (120, 521, 0.0021647809665476937)\n",
      "75 (67, 518, 0.002152315817028225)\n",
      "76 (14, 517, 0.0021481607671884023)\n",
      "77 (56, 505, 0.0020983001691105285)\n",
      "78 (108, 490, 0.002035974421513186)\n",
      "79 (121, 484, 0.002011044122474249)\n",
      "80 (16, 463, 0.0019237880758379696)\n",
      "81 (112, 452, 0.0018780825275999185)\n",
      "82 (96, 442, 0.0018365320292016902)\n",
      "83 (114, 433, 0.0017991365806432848)\n",
      "84 (90, 425, 0.0017658961819247022)\n",
      "85 (119, 424, 0.0017617411320848793)\n",
      "86 (60, 422, 0.0017534310324052336)\n",
      "87 (106, 418, 0.0017368108330459424)\n",
      "88 (83, 412, 0.0017118805340070053)\n",
      "89 (31, 403, 0.0016744850854485999)\n",
      "90 (100, 397, 0.001649554786409663)\n",
      "91 (89, 386, 0.0016038492381716118)\n",
      "92 (50, 381, 0.0015830739889724978)\n",
      "93 (109, 380, 0.001578918939132675)\n",
      "94 (59, 377, 0.0015664537896132064)\n",
      "95 (54, 376, 0.0015622987397733836)\n",
      "96 (53, 375, 0.0015581436899335607)\n",
      "97 (118, 375, 0.0015581436899335607)\n",
      "98 (44, 361, 0.0014999729921760412)\n",
      "99 (41, 354, 0.0014708876432972813)\n",
      "100 (79, 350, 0.00145426744393799)\n",
      "101 (99, 341, 0.0014168719953795846)\n",
      "102 (61, 338, 0.001404406845860116)\n",
      "103 (117, 328, 0.0013628563474618878)\n",
      "104 (94, 316, 0.0013129957493840138)\n",
      "105 (85, 295, 0.0012257397027477346)\n",
      "106 (88, 290, 0.0012049644535486203)\n",
      "107 (62, 288, 0.0011966543538689746)\n",
      "108 (113, 288, 0.0011966543538689746)\n",
      "109 (115, 282, 0.0011717240548300377)\n",
      "110 (68, 280, 0.001163413955150392)\n",
      "111 (92, 272, 0.0011301735564318094)\n",
      "112 (82, 264, 0.0010969331577132268)\n",
      "113 (37, 249, 0.0010346074101158843)\n",
      "114 (104, 247, 0.0010262973104362386)\n",
      "115 (38, 246, 0.001022142260596416)\n",
      "116 (40, 243, 0.0010096771110769474)\n",
      "117 (110, 242, 0.0010055220612371245)\n",
      "118 (72, 231, 0.0009598165129990734)\n",
      "119 (87, 225, 0.0009348862139601364)\n",
      "120 (101, 217, 0.0009016458152415538)\n",
      "121 (111, 212, 0.0008808705660424397)\n",
      "122 (105, 201, 0.0008351650178043885)\n"
     ]
    }
   ],
   "source": [
    "for idx, item in enumerate(sorted(label_freq, key=lambda x:x[1], reverse=True)):\n",
    "    print(idx+1, item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3932256067411528"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 前3占比\n",
    "sum([item[-1] for item in sorted(label_freq, key=lambda x:x[1], reverse=True)[:5]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
